Self Reported Quality Model

Looking at the data

Looks like there is a slightly higher rate of negative ratings for the high debt group, but also an even smaller increase in positive ratings.

data.frame(
  "High_debt_version" = d.both_completed %>%
    filter(high_debt_version == "true") %>%
    pull(quality_post_task) %>%
    revalue(c(
      "-3"="Very Bad",
      "-2"="Bad",
      "-1"="Somewhat Bad",
      "0"="Neutral",
      "1"="Somewhat Good",
      "2"="Good",
      "3"="Very Good"
    )),
  "Low_debt_version" = d.both_completed %>%
    filter(high_debt_version == "false") %>%
    pull(quality_post_task) %>%
    revalue(c(
      "-3"="Very Bad",
      "-2"="Bad",
      "-1"="Somewhat Bad",
      "0"="Neutral",
      "1"="Somewhat Good",
      "2"="Good",
      "3"="Very Good"
    ))
) %>%
  likert() %>%
  plot(
    type="density",
    facet = TRUE,
  )

Initial model

As the data is collected from a likert scale we will use a cumulative family.

We include high_debt_verison as a predictor in our model as this variable represent the very effect we want to measure. We also include a varying intercept for each individual to prevent the model from learning too much from single participants with extreme measurements.

Selecting Priors

We iterate over the model until we have sane priors.

own_quality.with <- extendable_model(
  base_name = "own_quality",
  base_formula = "quality_post_task ~ 1  + (1 | session)",
    base_priors = c(
    prior(normal(0, 2.5), class = "Intercept"),
    #prior(normal(0, 1), class = "b"),
    prior(exponential(1), class = "sd")
  ),
  family = cumulative(),
  data = d.both_completed,
  base_control = list(adapt_delta = 0.95)
)

Default priors

prior_summary(own_quality.with(only_priors= TRUE))

Selected priors

prior_summary(own_quality.with(sample_prior = "only"))

Prior Predictive Check

pp_check(own_quality.with(sample_prior = "only"), nsamples = 200, type = "bars")

Beta Parameter Influence

sim.size <- 1000
sim.intercept <- rnorm(sim.size, 0, 2.5)
sim.beta <- rnorm(sim.size, 0, 1)
sim.beta.diff <- (plogis(sim.intercept + sim.beta) / plogis(sim.intercept) * 100) - 100

data.frame(x = sim.beta.diff) %>%
  ggplot(aes(x)) +
  geom_density() +
  xlim(-150, 150) +
  labs(
    title = "Beta parameter prior influence",
    x = "Estimate with beta as % of estimate without beta",
    y = "Density"
  )

Model fit

We check the posterior distribution and can see that the model seems to have been able to fit the data well. Sampling seems to also have worked well as Rhat values are close to 1 and the sampling plots look nice.

Posterior Predictive check

pp_check(own_quality.with(), nsamples = 200, type = "bars")

Summary

summary(own_quality.with())

##  Family: cumulative 
##   Links: mu = logit; disc = identity 
## Formula: quality_post_task ~ 1 + (1 | session) 
##    Data: as.data.frame(data) (Number of observations: 44) 
## Samples: 4 chains, each with iter = 2000; warmup = 1000; thin = 1;
##          total post-warmup samples = 4000
## 
## Group-Level Effects: 
## ~session (Number of levels: 22) 
##               Estimate Est.Error l-95% CI u-95% CI Rhat Bulk_ESS Tail_ESS
## sd(Intercept)     0.52      0.40     0.02     1.48 1.00     1322     1851
## 
## Population-Level Effects: 
##              Estimate Est.Error l-95% CI u-95% CI Rhat Bulk_ESS Tail_ESS
## Intercept[1]    -3.26      0.76    -4.87    -1.98 1.00     2854     2211
## Intercept[2]    -0.85      0.36    -1.60    -0.16 1.00     3626     3194
## Intercept[3]     1.65      0.43     0.84     2.58 1.00     3782     2966
## Intercept[4]     2.37      0.52     1.44     3.47 1.00     4294     3480
## Intercept[5]     3.31      0.71     2.07     4.81 1.00     4956     3204
## 
## Family Specific Parameters: 
##      Estimate Est.Error l-95% CI u-95% CI Rhat Bulk_ESS Tail_ESS
## disc     1.00      0.00     1.00     1.00 1.00     4000     4000
## 
## Samples were drawn using sample(hmc). For each parameter, Bulk_ESS
## and Tail_ESS are effective sample size measures, and Rhat is the potential
## scale reduction factor on split chains (at convergence, Rhat = 1).

Sampling plots

plot(own_quality.with(), ask = FALSE)

Model quality extenstions

# default prior for monotonic predictor
edlvl_prior <- prior(dirichlet(2), class = "simo", coef = "moeducation_level1")
# deafult beta prior
beta_prior <- prior(normal(0, 1), class = "b")

We use loo to check some possible extensions on the model.

Variable names

loo_result <- loo(
  # Benchmark model(s)
  own_quality.with(),
  
  # New model(s)
  own_quality.with("var_names_new_good.ratio", beta_prior),
  own_quality.with("var_names_copied_good.ratio", beta_prior),
  own_quality.with(c("var_names_copied_good.ratio", "var_names_new_good.ratio"), beta_prior)
)

Comparison

loo_result[2]

## $diffs
##                                                                                                elpd_diff
## own_quality.with()                                                                              0.0     
## own_quality.with("var_names_copied_good.ratio", beta_prior)                                    -0.2     
## own_quality.with(c("var_names_copied_good.ratio", "var_names_new_good.ratio"),     beta_prior) -0.3     
## own_quality.with("var_names_new_good.ratio", beta_prior)                                       -0.4     
##                                                                                                se_diff
## own_quality.with()                                                                              0.0   
## own_quality.with("var_names_copied_good.ratio", beta_prior)                                     0.7   
## own_quality.with(c("var_names_copied_good.ratio", "var_names_new_good.ratio"),     beta_prior)  0.7   
## own_quality.with("var_names_new_good.ratio", beta_prior)                                        0.2

Diagnostics

loo_result[1]

## $loos
## $loos$`own_quality.with()`
## 
## Computed from 4000 by 44 log-likelihood matrix
## 
##          Estimate   SE
## elpd_loo    -62.2  6.8
## p_loo         8.2  1.6
## looic       124.4 13.6
## ------
## Monte Carlo SE of elpd_loo is 0.1.
## 
## Pareto k diagnostic values:
##                          Count Pct.    Min. n_eff
## (-Inf, 0.5]   (good)     34    77.3%   1043      
##  (0.5, 0.7]   (ok)       10    22.7%   497       
##    (0.7, 1]   (bad)       0     0.0%   <NA>      
##    (1, Inf)   (very bad)  0     0.0%   <NA>      
## 
## All Pareto k estimates are ok (k < 0.7).
## See help('pareto-k-diagnostic') for details.
## 
## $loos$`own_quality.with("var_names_new_good.ratio", beta_prior)`
## 
## Computed from 4000 by 44 log-likelihood matrix
## 
##          Estimate   SE
## elpd_loo    -62.6  6.8
## p_loo         8.7  1.6
## looic       125.3 13.6
## ------
## Monte Carlo SE of elpd_loo is NA.
## 
## Pareto k diagnostic values:
##                          Count Pct.    Min. n_eff
## (-Inf, 0.5]   (good)     40    90.9%   970       
##  (0.5, 0.7]   (ok)        3     6.8%   434       
##    (0.7, 1]   (bad)       1     2.3%   3394      
##    (1, Inf)   (very bad)  0     0.0%   <NA>      
## See help('pareto-k-diagnostic') for details.
## 
## $loos$`own_quality.with("var_names_copied_good.ratio", beta_prior)`
## 
## Computed from 4000 by 44 log-likelihood matrix
## 
##          Estimate   SE
## elpd_loo    -62.5  6.8
## p_loo         8.9  1.7
## looic       124.9 13.6
## ------
## Monte Carlo SE of elpd_loo is 0.1.
## 
## Pareto k diagnostic values:
##                          Count Pct.    Min. n_eff
## (-Inf, 0.5]   (good)     37    84.1%   1258      
##  (0.5, 0.7]   (ok)        7    15.9%   454       
##    (0.7, 1]   (bad)       0     0.0%   <NA>      
##    (1, Inf)   (very bad)  0     0.0%   <NA>      
## 
## All Pareto k estimates are ok (k < 0.7).
## See help('pareto-k-diagnostic') for details.
## 
## $loos$`own_quality.with(c("var_names_copied_good.ratio", "var_names_new_good.ratio"),     beta_prior)`
## 
## Computed from 4000 by 44 log-likelihood matrix
## 
##          Estimate   SE
## elpd_loo    -62.5  6.7
## p_loo         8.9  1.7
## looic       125.0 13.5
## ------
## Monte Carlo SE of elpd_loo is 0.1.
## 
## Pareto k diagnostic values:
##                          Count Pct.    Min. n_eff
## (-Inf, 0.5]   (good)     38    86.4%   1135      
##  (0.5, 0.7]   (ok)        6    13.6%   703       
##    (0.7, 1]   (bad)       0     0.0%   <NA>      
##    (1, Inf)   (very bad)  0     0.0%   <NA>      
## 
## All Pareto k estimates are ok (k < 0.7).
## See help('pareto-k-diagnostic') for details.

Reuse

loo_result <- loo(
  # Benchmark model(s)
  own_quality.with(),
  
  # New model(s)
  own_quality.with("reused_logic_constructor", beta_prior),
  own_quality.with("reused_logic_validation", beta_prior),
  own_quality.with(c("reused_logic_validation", "reused_logic_constructor"), beta_prior)
)

Comparison

loo_result[2]

## $diffs
##                                                                                            elpd_diff
## own_quality.with("reused_logic_validation", beta_prior)                                     0.0     
## own_quality.with(c("reused_logic_validation", "reused_logic_constructor"),     beta_prior) -0.4     
## own_quality.with("reused_logic_constructor", beta_prior)                                   -1.5     
## own_quality.with()                                                                         -2.2     
##                                                                                            se_diff
## own_quality.with("reused_logic_validation", beta_prior)                                     0.0   
## own_quality.with(c("reused_logic_validation", "reused_logic_constructor"),     beta_prior)  0.3   
## own_quality.with("reused_logic_constructor", beta_prior)                                    1.1   
## own_quality.with()                                                                          1.5

Diagnostics

loo_result[1]

## $loos
## $loos$`own_quality.with()`
## 
## Computed from 4000 by 44 log-likelihood matrix
## 
##          Estimate   SE
## elpd_loo    -62.2  6.8
## p_loo         8.2  1.6
## looic       124.4 13.6
## ------
## Monte Carlo SE of elpd_loo is 0.1.
## 
## Pareto k diagnostic values:
##                          Count Pct.    Min. n_eff
## (-Inf, 0.5]   (good)     34    77.3%   1043      
##  (0.5, 0.7]   (ok)       10    22.7%   497       
##    (0.7, 1]   (bad)       0     0.0%   <NA>      
##    (1, Inf)   (very bad)  0     0.0%   <NA>      
## 
## All Pareto k estimates are ok (k < 0.7).
## See help('pareto-k-diagnostic') for details.
## 
## $loos$`own_quality.with("reused_logic_constructor", beta_prior)`
## 
## Computed from 4000 by 44 log-likelihood matrix
## 
##          Estimate   SE
## elpd_loo    -61.5  6.5
## p_loo         8.8  1.6
## looic       123.0 13.0
## ------
## Monte Carlo SE of elpd_loo is 0.1.
## 
## Pareto k diagnostic values:
##                          Count Pct.    Min. n_eff
## (-Inf, 0.5]   (good)     39    88.6%   743       
##  (0.5, 0.7]   (ok)        5    11.4%   2158      
##    (0.7, 1]   (bad)       0     0.0%   <NA>      
##    (1, Inf)   (very bad)  0     0.0%   <NA>      
## 
## All Pareto k estimates are ok (k < 0.7).
## See help('pareto-k-diagnostic') for details.
## 
## $loos$`own_quality.with("reused_logic_validation", beta_prior)`
## 
## Computed from 4000 by 44 log-likelihood matrix
## 
##          Estimate   SE
## elpd_loo    -60.0  6.1
## p_loo         8.4  1.5
## looic       120.1 12.2
## ------
## Monte Carlo SE of elpd_loo is 0.1.
## 
## Pareto k diagnostic values:
##                          Count Pct.    Min. n_eff
## (-Inf, 0.5]   (good)     38    86.4%   1473      
##  (0.5, 0.7]   (ok)        6    13.6%   744       
##    (0.7, 1]   (bad)       0     0.0%   <NA>      
##    (1, Inf)   (very bad)  0     0.0%   <NA>      
## 
## All Pareto k estimates are ok (k < 0.7).
## See help('pareto-k-diagnostic') for details.
## 
## $loos$`own_quality.with(c("reused_logic_validation", "reused_logic_constructor"),     beta_prior)`
## 
## Computed from 4000 by 44 log-likelihood matrix
## 
##          Estimate   SE
## elpd_loo    -60.5  6.2
## p_loo         9.1  1.6
## looic       120.9 12.3
## ------
## Monte Carlo SE of elpd_loo is 0.1.
## 
## Pareto k diagnostic values:
##                          Count Pct.    Min. n_eff
## (-Inf, 0.5]   (good)     41    93.2%   859       
##  (0.5, 0.7]   (ok)        3     6.8%   3502      
##    (0.7, 1]   (bad)       0     0.0%   <NA>      
##    (1, Inf)   (very bad)  0     0.0%   <NA>      
## 
## All Pareto k estimates are ok (k < 0.7).
## See help('pareto-k-diagnostic') for details.

Utility methods

loo_result <- loo(
  # Benchmark model(s)
  own_quality.with(),
  
  # New model(s)
  own_quality.with("equals.exists", beta_prior),
  own_quality.with("hashcode.exists", beta_prior),
  own_quality.with(c("hashcode.state", "equals.exists"), beta_prior)
)

Comparison

loo_result[2]

## $diffs
##                                                                    elpd_diff
## own_quality.with()                                                  0.0     
## own_quality.with("equals.exists", beta_prior)                      -0.6     
## own_quality.with("hashcode.exists", beta_prior)                    -0.7     
## own_quality.with(c("hashcode.state", "equals.exists"), beta_prior) -0.7     
##                                                                    se_diff
## own_quality.with()                                                  0.0   
## own_quality.with("equals.exists", beta_prior)                       0.2   
## own_quality.with("hashcode.exists", beta_prior)                     0.2   
## own_quality.with(c("hashcode.state", "equals.exists"), beta_prior)  1.1

Diagnostics

loo_result[1]

## $loos
## $loos$`own_quality.with()`
## 
## Computed from 4000 by 44 log-likelihood matrix
## 
##          Estimate   SE
## elpd_loo    -62.2  6.8
## p_loo         8.2  1.6
## looic       124.4 13.6
## ------
## Monte Carlo SE of elpd_loo is 0.1.
## 
## Pareto k diagnostic values:
##                          Count Pct.    Min. n_eff
## (-Inf, 0.5]   (good)     34    77.3%   1043      
##  (0.5, 0.7]   (ok)       10    22.7%   497       
##    (0.7, 1]   (bad)       0     0.0%   <NA>      
##    (1, Inf)   (very bad)  0     0.0%   <NA>      
## 
## All Pareto k estimates are ok (k < 0.7).
## See help('pareto-k-diagnostic') for details.
## 
## $loos$`own_quality.with("equals.exists", beta_prior)`
## 
## Computed from 4000 by 44 log-likelihood matrix
## 
##          Estimate   SE
## elpd_loo    -62.8  6.9
## p_loo         9.0  1.8
## looic       125.7 13.7
## ------
## Monte Carlo SE of elpd_loo is 0.1.
## 
## Pareto k diagnostic values:
##                          Count Pct.    Min. n_eff
## (-Inf, 0.5]   (good)     38    86.4%   555       
##  (0.5, 0.7]   (ok)        6    13.6%   513       
##    (0.7, 1]   (bad)       0     0.0%   <NA>      
##    (1, Inf)   (very bad)  0     0.0%   <NA>      
## 
## All Pareto k estimates are ok (k < 0.7).
## See help('pareto-k-diagnostic') for details.
## 
## $loos$`own_quality.with("hashcode.exists", beta_prior)`
## 
## Computed from 4000 by 44 log-likelihood matrix
## 
##          Estimate   SE
## elpd_loo    -62.9  6.9
## p_loo         9.2  1.8
## looic       125.8 13.8
## ------
## Monte Carlo SE of elpd_loo is 0.1.
## 
## Pareto k diagnostic values:
##                          Count Pct.    Min. n_eff
## (-Inf, 0.5]   (good)     37    84.1%   1062      
##  (0.5, 0.7]   (ok)        7    15.9%   499       
##    (0.7, 1]   (bad)       0     0.0%   <NA>      
##    (1, Inf)   (very bad)  0     0.0%   <NA>      
## 
## All Pareto k estimates are ok (k < 0.7).
## See help('pareto-k-diagnostic') for details.
## 
## $loos$`own_quality.with(c("hashcode.state", "equals.exists"), beta_prior)`
## 
## Computed from 4000 by 44 log-likelihood matrix
## 
##          Estimate   SE
## elpd_loo    -62.9  6.7
## p_loo        10.3  1.9
## looic       125.9 13.5
## ------
## Monte Carlo SE of elpd_loo is NA.
## 
## Pareto k diagnostic values:
##                          Count Pct.    Min. n_eff
## (-Inf, 0.5]   (good)     40    90.9%   986       
##  (0.5, 0.7]   (ok)        3     6.8%   521       
##    (0.7, 1]   (bad)       1     2.3%   309       
##    (1, Inf)   (very bad)  0     0.0%   <NA>      
## See help('pareto-k-diagnostic') for details.

Other quality attributes

loo_result <- loo(
  # Benchmark model(s)
  own_quality.with(),
  
  # New model(s)
  own_quality.with("sonarqube_issues.s"),
  own_quality.with("documentation")
)

Comparison

loo_result[2]

## $diffs
##                                        elpd_diff se_diff
## own_quality.with()                      0.0       0.0   
## own_quality.with("sonarqube_issues.s") -1.7       1.1   
## own_quality.with("documentation")      -2.7       1.4

Diagnostics

loo_result[1]

## $loos
## $loos$`own_quality.with()`
## 
## Computed from 4000 by 44 log-likelihood matrix
## 
##          Estimate   SE
## elpd_loo    -62.2  6.8
## p_loo         8.2  1.6
## looic       124.4 13.6
## ------
## Monte Carlo SE of elpd_loo is 0.1.
## 
## Pareto k diagnostic values:
##                          Count Pct.    Min. n_eff
## (-Inf, 0.5]   (good)     34    77.3%   1043      
##  (0.5, 0.7]   (ok)       10    22.7%   497       
##    (0.7, 1]   (bad)       0     0.0%   <NA>      
##    (1, Inf)   (very bad)  0     0.0%   <NA>      
## 
## All Pareto k estimates are ok (k < 0.7).
## See help('pareto-k-diagnostic') for details.
## 
## $loos$`own_quality.with("sonarqube_issues.s")`
## 
## Computed from 4000 by 44 log-likelihood matrix
## 
##          Estimate   SE
## elpd_loo    -63.9  7.3
## p_loo        10.3  2.4
## looic       127.9 14.5
## ------
## Monte Carlo SE of elpd_loo is 0.1.
## 
## Pareto k diagnostic values:
##                          Count Pct.    Min. n_eff
## (-Inf, 0.5]   (good)     36    81.8%   1763      
##  (0.5, 0.7]   (ok)        8    18.2%   238       
##    (0.7, 1]   (bad)       0     0.0%   <NA>      
##    (1, Inf)   (very bad)  0     0.0%   <NA>      
## 
## All Pareto k estimates are ok (k < 0.7).
## See help('pareto-k-diagnostic') for details.
## 
## $loos$`own_quality.with("documentation")`
## 
## Computed from 4000 by 44 log-likelihood matrix
## 
##          Estimate   SE
## elpd_loo    -64.9  7.4
## p_loo        11.7  2.5
## looic       129.7 14.8
## ------
## Monte Carlo SE of elpd_loo is 0.1.
## 
## Pareto k diagnostic values:
##                          Count Pct.    Min. n_eff
## (-Inf, 0.5]   (good)     39    88.6%   618       
##  (0.5, 0.7]   (ok)        5    11.4%   315       
##    (0.7, 1]   (bad)       0     0.0%   <NA>      
##    (1, Inf)   (very bad)  0     0.0%   <NA>      
## 
## All Pareto k estimates are ok (k < 0.7).
## See help('pareto-k-diagnostic') for details.

Model predictor extenstions

We create a new base model based on what we learned from extending the previous model with different quality measurements. We take care to not include redundant quality indicators.

own_quality1.with <- extendable_model(
  base_name = "own_quality1",
  base_formula = "quality_post_task ~ 1 + 
    var_names_copied_good.ratio + 
    var_names_new_good.ratio + 
    reused_logic_validation + 
    equals.exists + 
    sonarqube_issues.s + 
    documentation + 
    (1 | session)",
    base_priors = c(
    prior(normal(0, 2.5), class = "Intercept"),
    prior(normal(0, 1), class = "b"),
    prior(exponential(1), class = "sd")
  ),
  family = cumulative(),
  data = data.frame(d.both_completed),
  base_control = list(adapt_delta = 0.95)
)

One variable

loo_result <- loo(
  # Benchmark model(s)
  own_quality.with(),
  own_quality1.with(),
  
  # New model(s)
  own_quality1.with("work_domain"),
  own_quality1.with("work_experience_programming.s"),
  own_quality1.with("work_experience_java.s"),
  own_quality1.with("education_field"),
  own_quality1.with("mo(education_level)", edlvl_prior),
  own_quality1.with("workplace_peer_review"),
  own_quality1.with("workplace_td_tracking"),
  own_quality1.with("workplace_pair_programming"),
  own_quality1.with("workplace_coding_standards"),
  own_quality1.with("scenario"),
  own_quality1.with("group")
)

Comparison

loo_result[2]

## $diffs
##                                                       elpd_diff se_diff
## own_quality1.with("mo(education_level)", edlvl_prior)  0.0       0.0   
## own_quality.with()                                    -1.1       3.0   
## own_quality1.with("education_field")                  -2.8       2.7   
## own_quality1.with("workplace_peer_review")            -3.2       2.3   
## own_quality1.with()                                   -3.2       2.6   
## own_quality1.with("workplace_coding_standards")       -3.5       2.4   
## own_quality1.with("group")                            -3.5       2.1   
## own_quality1.with("workplace_pair_programming")       -3.6       2.6   
## own_quality1.with("scenario")                         -3.8       2.6   
## own_quality1.with("workplace_td_tracking")            -3.8       2.2   
## own_quality1.with("work_experience_java.s")           -4.0       2.2   
## own_quality1.with("work_domain")                      -4.4       2.6   
## own_quality1.with("work_experience_programming.s")    -4.7       2.0

Diagnostics

loo_result[1]

## $loos
## $loos$`own_quality.with()`
## 
## Computed from 4000 by 44 log-likelihood matrix
## 
##          Estimate   SE
## elpd_loo    -62.2  6.8
## p_loo         8.2  1.6
## looic       124.4 13.6
## ------
## Monte Carlo SE of elpd_loo is 0.1.
## 
## Pareto k diagnostic values:
##                          Count Pct.    Min. n_eff
## (-Inf, 0.5]   (good)     34    77.3%   1043      
##  (0.5, 0.7]   (ok)       10    22.7%   497       
##    (0.7, 1]   (bad)       0     0.0%   <NA>      
##    (1, Inf)   (very bad)  0     0.0%   <NA>      
## 
## All Pareto k estimates are ok (k < 0.7).
## See help('pareto-k-diagnostic') for details.
## 
## $loos$`own_quality1.with()`
## 
## Computed from 4000 by 44 log-likelihood matrix
## 
##          Estimate   SE
## elpd_loo    -64.3  6.8
## p_loo        13.6  2.7
## looic       128.6 13.6
## ------
## Monte Carlo SE of elpd_loo is NA.
## 
## Pareto k diagnostic values:
##                          Count Pct.    Min. n_eff
## (-Inf, 0.5]   (good)     40    90.9%   661       
##  (0.5, 0.7]   (ok)        2     4.5%   1140      
##    (0.7, 1]   (bad)       2     4.5%   96        
##    (1, Inf)   (very bad)  0     0.0%   <NA>      
## See help('pareto-k-diagnostic') for details.
## 
## $loos$`own_quality1.with("work_domain")`
## 
## Computed from 4000 by 44 log-likelihood matrix
## 
##          Estimate   SE
## elpd_loo    -65.5  7.2
## p_loo        17.0  3.2
## looic       131.0 14.3
## ------
## Monte Carlo SE of elpd_loo is 0.1.
## 
## Pareto k diagnostic values:
##                          Count Pct.    Min. n_eff
## (-Inf, 0.5]   (good)     38    86.4%   504       
##  (0.5, 0.7]   (ok)        6    13.6%   249       
##    (0.7, 1]   (bad)       0     0.0%   <NA>      
##    (1, Inf)   (very bad)  0     0.0%   <NA>      
## 
## All Pareto k estimates are ok (k < 0.7).
## See help('pareto-k-diagnostic') for details.
## 
## $loos$`own_quality1.with("work_experience_programming.s")`
## 
## Computed from 4000 by 44 log-likelihood matrix
## 
##          Estimate   SE
## elpd_loo    -65.8  7.2
## p_loo        15.7  3.3
## looic       131.6 14.5
## ------
## Monte Carlo SE of elpd_loo is NA.
## 
## Pareto k diagnostic values:
##                          Count Pct.    Min. n_eff
## (-Inf, 0.5]   (good)     35    79.5%   467       
##  (0.5, 0.7]   (ok)        8    18.2%   195       
##    (0.7, 1]   (bad)       1     2.3%   137       
##    (1, Inf)   (very bad)  0     0.0%   <NA>      
## See help('pareto-k-diagnostic') for details.
## 
## $loos$`own_quality1.with("work_experience_java.s")`
## 
## Computed from 4000 by 44 log-likelihood matrix
## 
##          Estimate   SE
## elpd_loo    -65.1  6.9
## p_loo        14.7  2.9
## looic       130.2 13.7
## ------
## Monte Carlo SE of elpd_loo is NA.
## 
## Pareto k diagnostic values:
##                          Count Pct.    Min. n_eff
## (-Inf, 0.5]   (good)     34    77.3%   942       
##  (0.5, 0.7]   (ok)        9    20.5%   326       
##    (0.7, 1]   (bad)       1     2.3%   168       
##    (1, Inf)   (very bad)  0     0.0%   <NA>      
## See help('pareto-k-diagnostic') for details.
## 
## $loos$`own_quality1.with("education_field")`
## 
## Computed from 4000 by 44 log-likelihood matrix
## 
##          Estimate   SE
## elpd_loo    -63.9  6.7
## p_loo        13.9  2.5
## looic       127.7 13.4
## ------
## Monte Carlo SE of elpd_loo is NA.
## 
## Pareto k diagnostic values:
##                          Count Pct.    Min. n_eff
## (-Inf, 0.5]   (good)     40    90.9%   892       
##  (0.5, 0.7]   (ok)        3     6.8%   227       
##    (0.7, 1]   (bad)       1     2.3%   133       
##    (1, Inf)   (very bad)  0     0.0%   <NA>      
## See help('pareto-k-diagnostic') for details.
## 
## $loos$`own_quality1.with("mo(education_level)", edlvl_prior)`
## 
## Computed from 4000 by 44 log-likelihood matrix
## 
##          Estimate   SE
## elpd_loo    -61.1  7.2
## p_loo        12.9  2.6
## looic       122.1 14.4
## ------
## Monte Carlo SE of elpd_loo is 0.1.
## 
## Pareto k diagnostic values:
##                          Count Pct.    Min. n_eff
## (-Inf, 0.5]   (good)     42    95.5%   770       
##  (0.5, 0.7]   (ok)        2     4.5%   280       
##    (0.7, 1]   (bad)       0     0.0%   <NA>      
##    (1, Inf)   (very bad)  0     0.0%   <NA>      
## 
## All Pareto k estimates are ok (k < 0.7).
## See help('pareto-k-diagnostic') for details.
## 
## $loos$`own_quality1.with("workplace_peer_review")`
## 
## Computed from 4000 by 44 log-likelihood matrix
## 
##          Estimate   SE
## elpd_loo    -64.2  6.9
## p_loo        13.9  2.6
## looic       128.5 13.7
## ------
## Monte Carlo SE of elpd_loo is 0.1.
## 
## Pareto k diagnostic values:
##                          Count Pct.    Min. n_eff
## (-Inf, 0.5]   (good)     39    88.6%   935       
##  (0.5, 0.7]   (ok)        5    11.4%   241       
##    (0.7, 1]   (bad)       0     0.0%   <NA>      
##    (1, Inf)   (very bad)  0     0.0%   <NA>      
## 
## All Pareto k estimates are ok (k < 0.7).
## See help('pareto-k-diagnostic') for details.
## 
## $loos$`own_quality1.with("workplace_td_tracking")`
## 
## Computed from 4000 by 44 log-likelihood matrix
## 
##          Estimate   SE
## elpd_loo    -64.9  6.8
## p_loo        14.4  2.7
## looic       129.7 13.6
## ------
## Monte Carlo SE of elpd_loo is NA.
## 
## Pareto k diagnostic values:
##                          Count Pct.    Min. n_eff
## (-Inf, 0.5]   (good)     38    86.4%   981       
##  (0.5, 0.7]   (ok)        4     9.1%   350       
##    (0.7, 1]   (bad)       2     4.5%   191       
##    (1, Inf)   (very bad)  0     0.0%   <NA>      
## See help('pareto-k-diagnostic') for details.
## 
## $loos$`own_quality1.with("workplace_pair_programming")`
## 
## Computed from 4000 by 44 log-likelihood matrix
## 
##          Estimate   SE
## elpd_loo    -64.7  6.8
## p_loo        14.3  2.6
## looic       129.4 13.6
## ------
## Monte Carlo SE of elpd_loo is 0.1.
## 
## Pareto k diagnostic values:
##                          Count Pct.    Min. n_eff
## (-Inf, 0.5]   (good)     36    81.8%   938       
##  (0.5, 0.7]   (ok)        8    18.2%   191       
##    (0.7, 1]   (bad)       0     0.0%   <NA>      
##    (1, Inf)   (very bad)  0     0.0%   <NA>      
## 
## All Pareto k estimates are ok (k < 0.7).
## See help('pareto-k-diagnostic') for details.
## 
## $loos$`own_quality1.with("workplace_coding_standards")`
## 
## Computed from 4000 by 44 log-likelihood matrix
## 
##          Estimate   SE
## elpd_loo    -64.5  6.7
## p_loo        14.1  2.6
## looic       129.1 13.5
## ------
## Monte Carlo SE of elpd_loo is NA.
## 
## Pareto k diagnostic values:
##                          Count Pct.    Min. n_eff
## (-Inf, 0.5]   (good)     35    79.5%   590       
##  (0.5, 0.7]   (ok)        8    18.2%   265       
##    (0.7, 1]   (bad)       1     2.3%   173       
##    (1, Inf)   (very bad)  0     0.0%   <NA>      
## See help('pareto-k-diagnostic') for details.
## 
## $loos$`own_quality1.with("scenario")`
## 
## Computed from 4000 by 44 log-likelihood matrix
## 
##          Estimate   SE
## elpd_loo    -64.8  6.7
## p_loo        14.2  2.6
## looic       129.7 13.3
## ------
## Monte Carlo SE of elpd_loo is 0.1.
## 
## Pareto k diagnostic values:
##                          Count Pct.    Min. n_eff
## (-Inf, 0.5]   (good)     39    88.6%   582       
##  (0.5, 0.7]   (ok)        5    11.4%   204       
##    (0.7, 1]   (bad)       0     0.0%   <NA>      
##    (1, Inf)   (very bad)  0     0.0%   <NA>      
## 
## All Pareto k estimates are ok (k < 0.7).
## See help('pareto-k-diagnostic') for details.
## 
## $loos$`own_quality1.with("group")`
## 
## Computed from 4000 by 44 log-likelihood matrix
## 
##          Estimate   SE
## elpd_loo    -64.6  6.9
## p_loo        14.9  2.7
## looic       129.2 13.9
## ------
## Monte Carlo SE of elpd_loo is NA.
## 
## Pareto k diagnostic values:
##                          Count Pct.    Min. n_eff
## (-Inf, 0.5]   (good)     38    86.4%   538       
##  (0.5, 0.7]   (ok)        4     9.1%   568       
##    (0.7, 1]   (bad)       2     4.5%   258       
##    (1, Inf)   (very bad)  0     0.0%   <NA>      
## See help('pareto-k-diagnostic') for details.

Two variables

loo_result <- loo(
  # Benchmark model(s)
  own_quality.with(),
  own_quality1.with(),
  
  own_quality1.with("mo(education_level)", edlvl_prior),
  own_quality1.with("education_field"),
  own_quality1.with("workplace_peer_review"),
  
  # New model(s)
  own_quality1.with(c("mo(education_level)", "education_field"), edlvl_prior),
  own_quality1.with(c("mo(education_level)", "workplace_peer_review"), edlvl_prior),
  own_quality1.with(c("education_field", "workplace_peer_review"))
)

Comparison

loo_result[2]

## $diffs
##                                                                                       elpd_diff
## own_quality1.with("mo(education_level)", edlvl_prior)                                  0.0     
## own_quality1.with(c("mo(education_level)", "education_field"),     edlvl_prior)       -0.7     
## own_quality1.with(c("mo(education_level)", "workplace_peer_review"),     edlvl_prior) -0.8     
## own_quality.with()                                                                    -1.1     
## own_quality1.with("education_field")                                                  -2.8     
## own_quality1.with("workplace_peer_review")                                            -3.2     
## own_quality1.with()                                                                   -3.2     
## own_quality1.with(c("education_field", "workplace_peer_review"))                      -3.4     
##                                                                                       se_diff
## own_quality1.with("mo(education_level)", edlvl_prior)                                  0.0   
## own_quality1.with(c("mo(education_level)", "education_field"),     edlvl_prior)        0.4   
## own_quality1.with(c("mo(education_level)", "workplace_peer_review"),     edlvl_prior)  0.3   
## own_quality.with()                                                                     3.0   
## own_quality1.with("education_field")                                                   2.7   
## own_quality1.with("workplace_peer_review")                                             2.3   
## own_quality1.with()                                                                    2.6   
## own_quality1.with(c("education_field", "workplace_peer_review"))                       2.4

Diagnostics

loo_result[1]

## $loos
## $loos$`own_quality.with()`
## 
## Computed from 4000 by 44 log-likelihood matrix
## 
##          Estimate   SE
## elpd_loo    -62.2  6.8
## p_loo         8.2  1.6
## looic       124.4 13.6
## ------
## Monte Carlo SE of elpd_loo is 0.1.
## 
## Pareto k diagnostic values:
##                          Count Pct.    Min. n_eff
## (-Inf, 0.5]   (good)     34    77.3%   1043      
##  (0.5, 0.7]   (ok)       10    22.7%   497       
##    (0.7, 1]   (bad)       0     0.0%   <NA>      
##    (1, Inf)   (very bad)  0     0.0%   <NA>      
## 
## All Pareto k estimates are ok (k < 0.7).
## See help('pareto-k-diagnostic') for details.
## 
## $loos$`own_quality1.with()`
## 
## Computed from 4000 by 44 log-likelihood matrix
## 
##          Estimate   SE
## elpd_loo    -64.3  6.8
## p_loo        13.6  2.7
## looic       128.6 13.6
## ------
## Monte Carlo SE of elpd_loo is NA.
## 
## Pareto k diagnostic values:
##                          Count Pct.    Min. n_eff
## (-Inf, 0.5]   (good)     40    90.9%   661       
##  (0.5, 0.7]   (ok)        2     4.5%   1140      
##    (0.7, 1]   (bad)       2     4.5%   96        
##    (1, Inf)   (very bad)  0     0.0%   <NA>      
## See help('pareto-k-diagnostic') for details.
## 
## $loos$`own_quality1.with("mo(education_level)", edlvl_prior)`
## 
## Computed from 4000 by 44 log-likelihood matrix
## 
##          Estimate   SE
## elpd_loo    -61.1  7.2
## p_loo        12.9  2.6
## looic       122.1 14.4
## ------
## Monte Carlo SE of elpd_loo is 0.1.
## 
## Pareto k diagnostic values:
##                          Count Pct.    Min. n_eff
## (-Inf, 0.5]   (good)     42    95.5%   770       
##  (0.5, 0.7]   (ok)        2     4.5%   280       
##    (0.7, 1]   (bad)       0     0.0%   <NA>      
##    (1, Inf)   (very bad)  0     0.0%   <NA>      
## 
## All Pareto k estimates are ok (k < 0.7).
## See help('pareto-k-diagnostic') for details.
## 
## $loos$`own_quality1.with("education_field")`
## 
## Computed from 4000 by 44 log-likelihood matrix
## 
##          Estimate   SE
## elpd_loo    -63.9  6.7
## p_loo        13.9  2.5
## looic       127.7 13.4
## ------
## Monte Carlo SE of elpd_loo is NA.
## 
## Pareto k diagnostic values:
##                          Count Pct.    Min. n_eff
## (-Inf, 0.5]   (good)     40    90.9%   892       
##  (0.5, 0.7]   (ok)        3     6.8%   227       
##    (0.7, 1]   (bad)       1     2.3%   133       
##    (1, Inf)   (very bad)  0     0.0%   <NA>      
## See help('pareto-k-diagnostic') for details.
## 
## $loos$`own_quality1.with("workplace_peer_review")`
## 
## Computed from 4000 by 44 log-likelihood matrix
## 
##          Estimate   SE
## elpd_loo    -64.2  6.9
## p_loo        13.9  2.6
## looic       128.5 13.7
## ------
## Monte Carlo SE of elpd_loo is 0.1.
## 
## Pareto k diagnostic values:
##                          Count Pct.    Min. n_eff
## (-Inf, 0.5]   (good)     39    88.6%   935       
##  (0.5, 0.7]   (ok)        5    11.4%   241       
##    (0.7, 1]   (bad)       0     0.0%   <NA>      
##    (1, Inf)   (very bad)  0     0.0%   <NA>      
## 
## All Pareto k estimates are ok (k < 0.7).
## See help('pareto-k-diagnostic') for details.
## 
## $loos$`own_quality1.with(c("mo(education_level)", "education_field"),     edlvl_prior)`
## 
## Computed from 4000 by 44 log-likelihood matrix
## 
##          Estimate   SE
## elpd_loo    -61.8  7.2
## p_loo        14.1  2.7
## looic       123.6 14.4
## ------
## Monte Carlo SE of elpd_loo is NA.
## 
## Pareto k diagnostic values:
##                          Count Pct.    Min. n_eff
## (-Inf, 0.5]   (good)     39    88.6%   411       
##  (0.5, 0.7]   (ok)        3     6.8%   925       
##    (0.7, 1]   (bad)       2     4.5%   221       
##    (1, Inf)   (very bad)  0     0.0%   <NA>      
## See help('pareto-k-diagnostic') for details.
## 
## $loos$`own_quality1.with(c("mo(education_level)", "workplace_peer_review"),     edlvl_prior)`
## 
## Computed from 4000 by 44 log-likelihood matrix
## 
##          Estimate   SE
## elpd_loo    -61.8  7.3
## p_loo        13.8  2.8
## looic       123.7 14.7
## ------
## Monte Carlo SE of elpd_loo is NA.
## 
## Pareto k diagnostic values:
##                          Count Pct.    Min. n_eff
## (-Inf, 0.5]   (good)     37    84.1%   1056      
##  (0.5, 0.7]   (ok)        6    13.6%   394       
##    (0.7, 1]   (bad)       1     2.3%   161       
##    (1, Inf)   (very bad)  0     0.0%   <NA>      
## See help('pareto-k-diagnostic') for details.
## 
## $loos$`own_quality1.with(c("education_field", "workplace_peer_review"))`
## 
## Computed from 4000 by 44 log-likelihood matrix
## 
##          Estimate   SE
## elpd_loo    -64.5  6.8
## p_loo        14.7  2.6
## looic       128.9 13.7
## ------
## Monte Carlo SE of elpd_loo is NA.
## 
## Pareto k diagnostic values:
##                          Count Pct.    Min. n_eff
## (-Inf, 0.5]   (good)     39    88.6%   798       
##  (0.5, 0.7]   (ok)        4     9.1%   257       
##    (0.7, 1]   (bad)       1     2.3%   2782      
##    (1, Inf)   (very bad)  0     0.0%   <NA>      
## See help('pareto-k-diagnostic') for details.

Three variables

loo_result <- loo(
  # Benchmark model(s)
  own_quality.with(),
  own_quality1.with(),
  
  own_quality1.with("mo(education_level)", edlvl_prior),
  own_quality1.with("education_field"),
  own_quality1.with("workplace_peer_review"),
  
  own_quality1.with(c("mo(education_level)", "education_field"), edlvl_prior),
  own_quality1.with(c("mo(education_level)", "workplace_peer_review"), edlvl_prior),
  
  # New model(s)
  own_quality1.with(c("mo(education_level)", "education_field", "workplace_peer_review"), edlvl_prior)
)

Comparison

loo_result[2]

## $diffs
##                                                                                                          elpd_diff
## own_quality1.with("mo(education_level)", edlvl_prior)                                                     0.0     
## own_quality1.with(c("mo(education_level)", "education_field"),     edlvl_prior)                          -0.7     
## own_quality1.with(c("mo(education_level)", "workplace_peer_review"),     edlvl_prior)                    -0.8     
## own_quality.with()                                                                                       -1.1     
## own_quality1.with(c("mo(education_level)", "education_field",     "workplace_peer_review"), edlvl_prior) -1.3     
## own_quality1.with("education_field")                                                                     -2.8     
## own_quality1.with("workplace_peer_review")                                                               -3.2     
## own_quality1.with()                                                                                      -3.2     
##                                                                                                          se_diff
## own_quality1.with("mo(education_level)", edlvl_prior)                                                     0.0   
## own_quality1.with(c("mo(education_level)", "education_field"),     edlvl_prior)                           0.4   
## own_quality1.with(c("mo(education_level)", "workplace_peer_review"),     edlvl_prior)                     0.3   
## own_quality.with()                                                                                        3.0   
## own_quality1.with(c("mo(education_level)", "education_field",     "workplace_peer_review"), edlvl_prior)  0.5   
## own_quality1.with("education_field")                                                                      2.7   
## own_quality1.with("workplace_peer_review")                                                                2.3   
## own_quality1.with()                                                                                       2.6

Diagnostics

loo_result[1]

## $loos
## $loos$`own_quality.with()`
## 
## Computed from 4000 by 44 log-likelihood matrix
## 
##          Estimate   SE
## elpd_loo    -62.2  6.8
## p_loo         8.2  1.6
## looic       124.4 13.6
## ------
## Monte Carlo SE of elpd_loo is 0.1.
## 
## Pareto k diagnostic values:
##                          Count Pct.    Min. n_eff
## (-Inf, 0.5]   (good)     34    77.3%   1043      
##  (0.5, 0.7]   (ok)       10    22.7%   497       
##    (0.7, 1]   (bad)       0     0.0%   <NA>      
##    (1, Inf)   (very bad)  0     0.0%   <NA>      
## 
## All Pareto k estimates are ok (k < 0.7).
## See help('pareto-k-diagnostic') for details.
## 
## $loos$`own_quality1.with()`
## 
## Computed from 4000 by 44 log-likelihood matrix
## 
##          Estimate   SE
## elpd_loo    -64.3  6.8
## p_loo        13.6  2.7
## looic       128.6 13.6
## ------
## Monte Carlo SE of elpd_loo is NA.
## 
## Pareto k diagnostic values:
##                          Count Pct.    Min. n_eff
## (-Inf, 0.5]   (good)     40    90.9%   661       
##  (0.5, 0.7]   (ok)        2     4.5%   1140      
##    (0.7, 1]   (bad)       2     4.5%   96        
##    (1, Inf)   (very bad)  0     0.0%   <NA>      
## See help('pareto-k-diagnostic') for details.
## 
## $loos$`own_quality1.with("mo(education_level)", edlvl_prior)`
## 
## Computed from 4000 by 44 log-likelihood matrix
## 
##          Estimate   SE
## elpd_loo    -61.1  7.2
## p_loo        12.9  2.6
## looic       122.1 14.4
## ------
## Monte Carlo SE of elpd_loo is 0.1.
## 
## Pareto k diagnostic values:
##                          Count Pct.    Min. n_eff
## (-Inf, 0.5]   (good)     42    95.5%   770       
##  (0.5, 0.7]   (ok)        2     4.5%   280       
##    (0.7, 1]   (bad)       0     0.0%   <NA>      
##    (1, Inf)   (very bad)  0     0.0%   <NA>      
## 
## All Pareto k estimates are ok (k < 0.7).
## See help('pareto-k-diagnostic') for details.
## 
## $loos$`own_quality1.with("education_field")`
## 
## Computed from 4000 by 44 log-likelihood matrix
## 
##          Estimate   SE
## elpd_loo    -63.9  6.7
## p_loo        13.9  2.5
## looic       127.7 13.4
## ------
## Monte Carlo SE of elpd_loo is NA.
## 
## Pareto k diagnostic values:
##                          Count Pct.    Min. n_eff
## (-Inf, 0.5]   (good)     40    90.9%   892       
##  (0.5, 0.7]   (ok)        3     6.8%   227       
##    (0.7, 1]   (bad)       1     2.3%   133       
##    (1, Inf)   (very bad)  0     0.0%   <NA>      
## See help('pareto-k-diagnostic') for details.
## 
## $loos$`own_quality1.with("workplace_peer_review")`
## 
## Computed from 4000 by 44 log-likelihood matrix
## 
##          Estimate   SE
## elpd_loo    -64.2  6.9
## p_loo        13.9  2.6
## looic       128.5 13.7
## ------
## Monte Carlo SE of elpd_loo is 0.1.
## 
## Pareto k diagnostic values:
##                          Count Pct.    Min. n_eff
## (-Inf, 0.5]   (good)     39    88.6%   935       
##  (0.5, 0.7]   (ok)        5    11.4%   241       
##    (0.7, 1]   (bad)       0     0.0%   <NA>      
##    (1, Inf)   (very bad)  0     0.0%   <NA>      
## 
## All Pareto k estimates are ok (k < 0.7).
## See help('pareto-k-diagnostic') for details.
## 
## $loos$`own_quality1.with(c("mo(education_level)", "education_field"),     edlvl_prior)`
## 
## Computed from 4000 by 44 log-likelihood matrix
## 
##          Estimate   SE
## elpd_loo    -61.8  7.2
## p_loo        14.1  2.7
## looic       123.6 14.4
## ------
## Monte Carlo SE of elpd_loo is NA.
## 
## Pareto k diagnostic values:
##                          Count Pct.    Min. n_eff
## (-Inf, 0.5]   (good)     39    88.6%   411       
##  (0.5, 0.7]   (ok)        3     6.8%   925       
##    (0.7, 1]   (bad)       2     4.5%   221       
##    (1, Inf)   (very bad)  0     0.0%   <NA>      
## See help('pareto-k-diagnostic') for details.
## 
## $loos$`own_quality1.with(c("mo(education_level)", "workplace_peer_review"),     edlvl_prior)`
## 
## Computed from 4000 by 44 log-likelihood matrix
## 
##          Estimate   SE
## elpd_loo    -61.8  7.3
## p_loo        13.8  2.8
## looic       123.7 14.7
## ------
## Monte Carlo SE of elpd_loo is NA.
## 
## Pareto k diagnostic values:
##                          Count Pct.    Min. n_eff
## (-Inf, 0.5]   (good)     37    84.1%   1056      
##  (0.5, 0.7]   (ok)        6    13.6%   394       
##    (0.7, 1]   (bad)       1     2.3%   161       
##    (1, Inf)   (very bad)  0     0.0%   <NA>      
## See help('pareto-k-diagnostic') for details.
## 
## $loos$`own_quality1.with(c("mo(education_level)", "education_field",     "workplace_peer_review"), edlvl_prior)`
## 
## Computed from 4000 by 44 log-likelihood matrix
## 
##          Estimate   SE
## elpd_loo    -62.4  7.3
## p_loo        14.6  2.8
## looic       124.8 14.7
## ------
## Monte Carlo SE of elpd_loo is 0.1.
## 
## Pareto k diagnostic values:
##                          Count Pct.    Min. n_eff
## (-Inf, 0.5]   (good)     40    90.9%   1106      
##  (0.5, 0.7]   (ok)        4     9.1%   175       
##    (0.7, 1]   (bad)       0     0.0%   <NA>      
##    (1, Inf)   (very bad)  0     0.0%   <NA>      
## 
## All Pareto k estimates are ok (k < 0.7).
## See help('pareto-k-diagnostic') for details.

Candidate models

We pick some of our top performing models as candidates and inspect them closer.

The candidate models are named and listed in order of complexity.

OwnQuality0

We select the simplest model as a baseline.

own_quality0 <- brm(
  "quality_post_task ~ 1 + 
    var_names_copied_good.ratio + 
    var_names_new_good.ratio + 
    reused_logic_validation + 
    equals.exists + 
    sonarqube_issues.s + 
    documentation + 
    (1 | session)",
  prior = c(
    prior(normal(0, 2.5), class = "Intercept"),
    prior(normal(0, 1), class = "b"),
    prior(exponential(1), class = "sd")
  ),
  family = cumulative(),
  data = as.data.frame(d.both_completed),
  file = "fits/own_quality0",
  file_refit = "on_change",
  control = list(adapt_delta = 0.95),
  seed = 20210421
)

Summary

summary(own_quality0)

##  Family: cumulative 
##   Links: mu = logit; disc = identity 
## Formula: quality_post_task ~ 1 + var_names_copied_good.ratio + var_names_new_good.ratio + reused_logic_validation + equals.exists + sonarqube_issues.s + documentation + (1 | session) 
##    Data: as.data.frame(d.both_completed) (Number of observations: 44) 
## Samples: 4 chains, each with iter = 2000; warmup = 1000; thin = 1;
##          total post-warmup samples = 4000
## 
## Group-Level Effects: 
## ~session (Number of levels: 22) 
##               Estimate Est.Error l-95% CI u-95% CI Rhat Bulk_ESS Tail_ESS
## sd(Intercept)     0.55      0.42     0.02     1.57 1.00     1597     2169
## 
## Population-Level Effects: 
##                              Estimate Est.Error l-95% CI u-95% CI Rhat Bulk_ESS
## Intercept[1]                    -4.16      1.21    -6.61    -1.84 1.00     4785
## Intercept[2]                    -1.64      1.04    -3.69     0.43 1.00     5536
## Intercept[3]                     1.08      1.04    -0.91     3.23 1.00     5148
## Intercept[4]                     1.86      1.06    -0.15     4.06 1.00     5386
## Intercept[5]                     2.84      1.15     0.66     5.25 1.00     5107
## var_names_copied_good.ratio      0.18      0.69    -1.16     1.51 1.00     5238
## var_names_new_good.ratio        -0.19      0.76    -1.66     1.31 1.00     5848
## reused_logic_validationfalse    -1.29      0.69    -2.65     0.07 1.00     5620
## equals.existsFALSE              -0.12      0.60    -1.30     1.05 1.00     5595
## sonarqube_issues.s               0.14      0.33    -0.55     0.77 1.00     5153
## documentationIncorrect          -0.08      0.70    -1.45     1.33 1.00     4948
## documentationNone               -0.32      0.67    -1.61     1.00 1.00     5163
##                              Tail_ESS
## Intercept[1]                     2890
## Intercept[2]                     3421
## Intercept[3]                     3291
## Intercept[4]                     2967
## Intercept[5]                     3177
## var_names_copied_good.ratio      3458
## var_names_new_good.ratio         3368
## reused_logic_validationfalse     3143
## equals.existsFALSE               2682
## sonarqube_issues.s               3093
## documentationIncorrect           3273
## documentationNone                3166
## 
## Family Specific Parameters: 
##      Estimate Est.Error l-95% CI u-95% CI Rhat Bulk_ESS Tail_ESS
## disc     1.00      0.00     1.00     1.00 1.00     4000     4000
## 
## Samples were drawn using sample(hmc). For each parameter, Bulk_ESS
## and Tail_ESS are effective sample size measures, and Rhat is the potential
## scale reduction factor on split chains (at convergence, Rhat = 1).

Random effects

ranef(own_quality0)

## $session
## , , Intercept
## 
##                              Estimate Est.Error       Q2.5     Q97.5
## 6033d69a5af2c702367b3a95 -0.003597013 0.5691377 -1.2575950 1.2396757
## 6033d90a5af2c702367b3a96  0.014789321 0.5393521 -1.1573882 1.2094220
## 6034fc165af2c702367b3a98  0.217146642 0.6379817 -0.9111772 1.8831193
## 603500725af2c702367b3a99  0.217880628 0.6287807 -0.9036990 1.7542683
## 603f97625af2c702367b3a9d  0.195295978 0.6159302 -0.9047822 1.7731190
## 603fd5d95af2c702367b3a9e  0.211933782 0.6094602 -0.8348789 1.8124290
## 60409b7b5af2c702367b3a9f -0.291503792 0.6534897 -1.9900278 0.7329610
## 604b82b5a7718fbed181b336 -0.118956619 0.5970882 -1.5933948 1.0402783
## 6050c1bf856f36729d2e5218 -0.381506746 0.7043987 -2.1840885 0.5912000
## 6050e1e7856f36729d2e5219  0.300178134 0.6807822 -0.7088807 2.1312623
## 6055fdc6856f36729d2e521b -0.095024017 0.5870477 -1.4352780 1.0614585
## 60589862856f36729d2e521f  0.017417753 0.6351805 -1.3912627 1.4370708
## 605afa3a856f36729d2e5222  0.111599771 0.5979707 -1.0760257 1.5341443
## 605c8bc6856f36729d2e5223  0.115222936 0.5451251 -0.9293503 1.4080110
## 605f3f2d856f36729d2e5224 -0.294684438 0.6694775 -2.0817340 0.6980392
## 605f46c3856f36729d2e5225 -0.454960073 0.7527216 -2.4583685 0.4801064
## 60605337856f36729d2e5226 -0.042958797 0.5984055 -1.3864395 1.1934620
## 60609ae6856f36729d2e5228 -0.107330078 0.5570589 -1.4428330 0.9695812
## 6061ce91856f36729d2e522e  0.265634212 0.6361855 -0.7333939 1.8960458
## 6061f106856f36729d2e5231  0.024195154 0.5225656 -1.1240073 1.2148825
## 6068ea9f856f36729d2e523e  0.225063116 0.6117656 -0.7948535 1.8227282
## 6075ab05856f36729d2e5247 -0.459722059 0.7565080 -2.4875737 0.4958027

Sampling plots

plot(own_quality0, ask = FALSE)

Posterior predictive check

pp_check(own_quality0, nsamples = 200, type = "bars")

OwnQuality1

We select the best performing model with one variable.

own_quality1 <- brm(
  "quality_post_task ~ 1 + 
    var_names_copied_good.ratio + 
    var_names_new_good.ratio + 
    reused_logic_validation + 
    equals.exists + 
    sonarqube_issues.s + 
    documentation + 
    (1 | session) +
    mo(education_level)",
  prior = c(
    prior(normal(0, 2.5), class = "Intercept"),
    prior(normal(0, 1), class = "b"),
    prior(exponential(1), class = "sd"),
    prior(dirichlet(2), class = "simo", coef = "moeducation_level1")
  ),
  family = cumulative(),
  data = as.data.frame(d.both_completed),
  file = "fits/own_quality1",
  file_refit = "on_change",
  control = list(adapt_delta = 0.95),
  seed = 20210421
)

Summary

summary(own_quality1)

##  Family: cumulative 
##   Links: mu = logit; disc = identity 
## Formula: quality_post_task ~ 1 + var_names_copied_good.ratio + var_names_new_good.ratio + reused_logic_validation + equals.exists + sonarqube_issues.s + documentation + (1 | session) + mo(education_level) 
##    Data: as.data.frame(d.both_completed) (Number of observations: 44) 
## Samples: 4 chains, each with iter = 2000; warmup = 1000; thin = 1;
##          total post-warmup samples = 4000
## 
## Group-Level Effects: 
## ~session (Number of levels: 22) 
##               Estimate Est.Error l-95% CI u-95% CI Rhat Bulk_ESS Tail_ESS
## sd(Intercept)     0.41      0.34     0.01     1.27 1.00     2197     2087
## 
## Population-Level Effects: 
##                              Estimate Est.Error l-95% CI u-95% CI Rhat Bulk_ESS
## Intercept[1]                    -5.53      1.34    -8.30    -3.06 1.00     5225
## Intercept[2]                    -2.99      1.17    -5.29    -0.76 1.00     5526
## Intercept[3]                     0.05      1.12    -2.10     2.23 1.00     6353
## Intercept[4]                     0.90      1.14    -1.29     3.13 1.00     6385
## Intercept[5]                     1.94      1.23    -0.41     4.44 1.00     6717
## var_names_copied_good.ratio      0.31      0.69    -1.02     1.66 1.00     6468
## var_names_new_good.ratio        -0.21      0.74    -1.61     1.23 1.00     6675
## reused_logic_validationfalse    -1.14      0.68    -2.45     0.22 1.00     7016
## equals.existsFALSE               0.01      0.59    -1.14     1.17 1.00     6521
## sonarqube_issues.s               0.33      0.33    -0.35     0.94 1.00     6424
## documentationIncorrect          -0.22      0.72    -1.63     1.18 1.00     6196
## documentationNone               -0.36      0.68    -1.71     0.99 1.00     5939
## moeducation_level               -0.62      0.25    -1.10    -0.14 1.00     4139
##                              Tail_ESS
## Intercept[1]                     3039
## Intercept[2]                     3164
## Intercept[3]                     3301
## Intercept[4]                     3551
## Intercept[5]                     3625
## var_names_copied_good.ratio      3244
## var_names_new_good.ratio         3271
## reused_logic_validationfalse     3386
## equals.existsFALSE               3076
## sonarqube_issues.s               3303
## documentationIncorrect           2935
## documentationNone                2762
## moeducation_level                2796
## 
## Simplex Parameters: 
##                       Estimate Est.Error l-95% CI u-95% CI Rhat Bulk_ESS
## moeducation_level1[1]     0.22      0.12     0.03     0.49 1.00     8404
## moeducation_level1[2]     0.29      0.15     0.05     0.61 1.00     7341
## moeducation_level1[3]     0.24      0.14     0.04     0.55 1.00     6974
## moeducation_level1[4]     0.26      0.14     0.04     0.56 1.00     6785
##                       Tail_ESS
## moeducation_level1[1]     2696
## moeducation_level1[2]     2904
## moeducation_level1[3]     2608
## moeducation_level1[4]     2647
## 
## Family Specific Parameters: 
##      Estimate Est.Error l-95% CI u-95% CI Rhat Bulk_ESS Tail_ESS
## disc     1.00      0.00     1.00     1.00 1.00     4000     4000
## 
## Samples were drawn using sample(hmc). For each parameter, Bulk_ESS
## and Tail_ESS are effective sample size measures, and Rhat is the potential
## scale reduction factor on split chains (at convergence, Rhat = 1).

Random effects

ranef(own_quality1)

## $session
## , , Intercept
## 
##                             Estimate Est.Error       Q2.5     Q97.5
## 6033d69a5af2c702367b3a95  0.06345376 0.4485564 -0.8276452 1.1370718
## 6033d90a5af2c702367b3a96 -0.05426508 0.4674512 -1.1435667 0.8955601
## 6034fc165af2c702367b3a98  0.18612804 0.5230054 -0.6624199 1.5735883
## 603500725af2c702367b3a99  0.06066618 0.4929435 -0.9108607 1.2559880
## 603f97625af2c702367b3a9d  0.15770562 0.5074407 -0.6919706 1.4723548
## 603fd5d95af2c702367b3a9e  0.08487268 0.4619562 -0.7802987 1.2096927
## 60409b7b5af2c702367b3a9f -0.16826729 0.5084530 -1.4643742 0.6388627
## 604b82b5a7718fbed181b336  0.02327163 0.4754285 -0.9840679 1.1390013
## 6050c1bf856f36729d2e5218 -0.19337378 0.5084320 -1.5603862 0.5841307
## 6050e1e7856f36729d2e5219  0.14032548 0.5088986 -0.7286702 1.4890807
## 6055fdc6856f36729d2e521b -0.07091407 0.4677973 -1.2335113 0.8417580
## 60589862856f36729d2e521f  0.03895203 0.5036893 -1.0144965 1.2125065
## 605afa3a856f36729d2e5222  0.15054573 0.5138011 -0.7006577 1.4565383
## 605c8bc6856f36729d2e5223  0.20344464 0.5116691 -0.5863046 1.5952393
## 605f3f2d856f36729d2e5224 -0.13638148 0.4970087 -1.3862195 0.7279973
## 605f46c3856f36729d2e5225 -0.15981704 0.5134788 -1.5232868 0.7281112
## 60605337856f36729d2e5226  0.02742256 0.4771778 -1.0105487 1.0763487
## 60609ae6856f36729d2e5228 -0.07947723 0.4624653 -1.2304770 0.8277921
## 6061ce91856f36729d2e522e  0.12274911 0.4946236 -0.7876618 1.3457835
## 6061f106856f36729d2e5231 -0.06599802 0.4681963 -1.1701008 0.8540004
## 6068ea9f856f36729d2e523e  0.03282397 0.4496302 -0.8559524 1.0725408
## 6075ab05856f36729d2e5247 -0.18843313 0.5089148 -1.5473897 0.6357123

Sampling plots

plot(own_quality1, ask = FALSE)

Posterior predictive check

pp_check(own_quality1, nsamples = 200, type = "bars")

OwnQuality2

We select the best performing model with two variables.

own_quality2 <- brm(
  "quality_post_task ~ 1 + 
    var_names_copied_good.ratio + 
    var_names_new_good.ratio + 
    reused_logic_validation + 
    equals.exists + 
    sonarqube_issues.s + 
    documentation + 
    (1 | session) +
    mo(education_level) +
    education_field",
  prior = c(
    prior(normal(0, 2.5), class = "Intercept"),
    prior(normal(0, 1), class = "b"),
    prior(exponential(1), class = "sd"),
    prior(dirichlet(2), class = "simo", coef = "moeducation_level1")
  ),
  family = cumulative(),
  data = as.data.frame(d.both_completed),
  file = "fits/own_quality2",
  file_refit = "on_change",
  control = list(adapt_delta = 0.95),
  seed = 20210421
)

Summary

summary(own_quality2)

##  Family: cumulative 
##   Links: mu = logit; disc = identity 
## Formula: quality_post_task ~ 1 + var_names_copied_good.ratio + var_names_new_good.ratio + reused_logic_validation + equals.exists + sonarqube_issues.s + documentation + (1 | session) + mo(education_level) + education_field 
##    Data: as.data.frame(d.both_completed) (Number of observations: 44) 
## Samples: 4 chains, each with iter = 2000; warmup = 1000; thin = 1;
##          total post-warmup samples = 4000
## 
## Group-Level Effects: 
## ~session (Number of levels: 22) 
##               Estimate Est.Error l-95% CI u-95% CI Rhat Bulk_ESS Tail_ESS
## sd(Intercept)     0.43      0.35     0.01     1.30 1.00     2009     2255
## 
## Population-Level Effects: 
##                                    Estimate Est.Error l-95% CI u-95% CI Rhat
## Intercept[1]                          -5.39      1.40    -8.17    -2.68 1.00
## Intercept[2]                          -2.81      1.25    -5.26    -0.33 1.00
## Intercept[3]                           0.28      1.22    -2.04     2.65 1.00
## Intercept[4]                           1.14      1.23    -1.19     3.54 1.00
## Intercept[5]                           2.17      1.31    -0.31     4.76 1.00
## var_names_copied_good.ratio            0.29      0.69    -1.05     1.67 1.00
## var_names_new_good.ratio              -0.14      0.77    -1.64     1.34 1.00
## reused_logic_validationfalse          -1.15      0.67    -2.42     0.14 1.00
## equals.existsFALSE                     0.03      0.58    -1.08     1.20 1.00
## sonarqube_issues.s                     0.33      0.33    -0.33     0.96 1.00
## documentationIncorrect                -0.24      0.71    -1.65     1.15 1.00
## documentationNone                     -0.33      0.66    -1.64     0.96 1.00
## education_fieldInteractionDesign      -0.56      0.84    -2.19     1.11 1.00
## education_fieldNone                    0.06      0.85    -1.58     1.75 1.00
## education_fieldSoftwareEngineering     0.18      0.59    -0.99     1.31 1.00
## moeducation_level                     -0.60      0.25    -1.11    -0.12 1.00
##                                    Bulk_ESS Tail_ESS
## Intercept[1]                           4188     2961
## Intercept[2]                           4680     3129
## Intercept[3]                           5223     3231
## Intercept[4]                           5385     3378
## Intercept[5]                           5708     3296
## var_names_copied_good.ratio            5565     3269
## var_names_new_good.ratio               5338     3162
## reused_logic_validationfalse           6004     3163
## equals.existsFALSE                     6160     2950
## sonarqube_issues.s                     5660     3112
## documentationIncorrect                 5071     2902
## documentationNone                      5404     3087
## education_fieldInteractionDesign       7348     3272
## education_fieldNone                    6628     2752
## education_fieldSoftwareEngineering     6281     3545
## moeducation_level                      3832     3028
## 
## Simplex Parameters: 
##                       Estimate Est.Error l-95% CI u-95% CI Rhat Bulk_ESS
## moeducation_level1[1]     0.22      0.12     0.04     0.51 1.00     5585
## moeducation_level1[2]     0.28      0.15     0.04     0.61 1.00     6360
## moeducation_level1[3]     0.24      0.13     0.04     0.54 1.00     6496
## moeducation_level1[4]     0.26      0.14     0.04     0.56 1.00     5996
##                       Tail_ESS
## moeducation_level1[1]     2578
## moeducation_level1[2]     2651
## moeducation_level1[3]     2918
## moeducation_level1[4]     2952
## 
## Family Specific Parameters: 
##      Estimate Est.Error l-95% CI u-95% CI Rhat Bulk_ESS Tail_ESS
## disc     1.00      0.00     1.00     1.00 1.00     4000     4000
## 
## Samples were drawn using sample(hmc). For each parameter, Bulk_ESS
## and Tail_ESS are effective sample size measures, and Rhat is the potential
## scale reduction factor on split chains (at convergence, Rhat = 1).

Random effects

ranef(own_quality2)

## $session
## , , Intercept
## 
##                             Estimate Est.Error       Q2.5     Q97.5
## 6033d69a5af2c702367b3a95  0.05862681 0.4857455 -0.9121941 1.1835278
## 6033d90a5af2c702367b3a96 -0.07271323 0.4741916 -1.2154645 0.8372287
## 6034fc165af2c702367b3a98  0.19688443 0.5438811 -0.6333864 1.6680245
## 603500725af2c702367b3a99  0.06398798 0.4689289 -0.8765425 1.1960245
## 603f97625af2c702367b3a9d  0.16181150 0.5254654 -0.7154987 1.5184058
## 603fd5d95af2c702367b3a9e  0.07312910 0.4579353 -0.8192655 1.1168413
## 60409b7b5af2c702367b3a9f -0.17697272 0.5095151 -1.4989220 0.6204594
## 604b82b5a7718fbed181b336 -0.01560080 0.4669579 -1.0608800 1.0319015
## 6050c1bf856f36729d2e5218 -0.20436839 0.5376292 -1.6021532 0.6587857
## 6050e1e7856f36729d2e5219  0.16060710 0.5382503 -0.7046126 1.6168415
## 6055fdc6856f36729d2e521b -0.08492602 0.4881045 -1.2843642 0.8441007
## 60589862856f36729d2e521f  0.04474436 0.5209253 -0.9480323 1.2590313
## 605afa3a856f36729d2e5222  0.16971821 0.5315386 -0.7237819 1.5431340
## 605c8bc6856f36729d2e5223  0.20534044 0.5243361 -0.5873331 1.6182595
## 605f3f2d856f36729d2e5224 -0.14280855 0.5054777 -1.4510252 0.7291005
## 605f46c3856f36729d2e5225 -0.18680275 0.5255168 -1.5663795 0.6696721
## 60605337856f36729d2e5226  0.01504140 0.4987200 -1.0549720 1.1020085
## 60609ae6856f36729d2e5228 -0.07851222 0.4903465 -1.2815460 0.8457675
## 6061ce91856f36729d2e522e  0.12313348 0.4963280 -0.7650773 1.3863915
## 6061f106856f36729d2e5231 -0.06291447 0.4714056 -1.1808842 0.8799728
## 6068ea9f856f36729d2e523e  0.03559989 0.5030245 -1.0206005 1.2449750
## 6075ab05856f36729d2e5247 -0.12962967 0.5227119 -1.4453770 0.8034020

Sampling plots

plot(own_quality2, ask = FALSE)

Posterior predictive check

pp_check(own_quality2, nsamples = 200, type = "bars")

OwnQuality3

We select the best performing model with three variables.

own_quality3 <- brm(
  "quality_post_task ~ 1 + 
    var_names_copied_good.ratio + 
    var_names_new_good.ratio + 
    reused_logic_validation + 
    equals.exists + 
    sonarqube_issues.s + 
    documentation + 
    (1 | session) +
    mo(education_level) +
    education_field +
    workplace_peer_review",
  prior = c(
    prior(normal(0, 2.5), class = "Intercept"),
    prior(normal(0, 1), class = "b"),
    prior(exponential(1), class = "sd"),
    prior(dirichlet(2), class = "simo", coef = "moeducation_level1")
  ),
  family = cumulative(),
  data = as.data.frame(d.both_completed),
  file = "fits/own_quality3",
  file_refit = "on_change",
  control = list(adapt_delta = 0.95),
  seed = 20210421
)

Summary

summary(own_quality3)

##  Family: cumulative 
##   Links: mu = logit; disc = identity 
## Formula: quality_post_task ~ 1 + var_names_copied_good.ratio + var_names_new_good.ratio + reused_logic_validation + equals.exists + sonarqube_issues.s + documentation + (1 | session) + mo(education_level) + education_field + workplace_peer_review 
##    Data: as.data.frame(d.both_completed) (Number of observations: 44) 
## Samples: 4 chains, each with iter = 2000; warmup = 1000; thin = 1;
##          total post-warmup samples = 4000
## 
## Group-Level Effects: 
## ~session (Number of levels: 22) 
##               Estimate Est.Error l-95% CI u-95% CI Rhat Bulk_ESS Tail_ESS
## sd(Intercept)     0.44      0.36     0.02     1.32 1.00     1874     2570
## 
## Population-Level Effects: 
##                                    Estimate Est.Error l-95% CI u-95% CI Rhat
## Intercept[1]                          -5.42      1.45    -8.29    -2.67 1.00
## Intercept[2]                          -2.86      1.31    -5.54    -0.28 1.00
## Intercept[3]                           0.24      1.23    -2.22     2.67 1.00
## Intercept[4]                           1.10      1.26    -1.41     3.57 1.00
## Intercept[5]                           2.16      1.30    -0.48     4.78 1.00
## var_names_copied_good.ratio            0.31      0.68    -1.01     1.63 1.00
## var_names_new_good.ratio              -0.15      0.77    -1.67     1.36 1.00
## reused_logic_validationfalse          -1.15      0.71    -2.55     0.25 1.00
## equals.existsFALSE                     0.03      0.60    -1.16     1.21 1.00
## sonarqube_issues.s                     0.35      0.34    -0.33     0.98 1.00
## documentationIncorrect                -0.24      0.73    -1.67     1.19 1.00
## documentationNone                     -0.32      0.67    -1.64     0.99 1.00
## education_fieldInteractionDesign      -0.54      0.84    -2.19     1.11 1.00
## education_fieldNone                    0.06      0.86    -1.64     1.71 1.00
## education_fieldSoftwareEngineering     0.19      0.59    -0.97     1.34 1.00
## workplace_peer_reviewfalse            -0.09      0.59    -1.26     1.08 1.00
## moeducation_level                     -0.60      0.26    -1.13    -0.11 1.00
##                                    Bulk_ESS Tail_ESS
## Intercept[1]                           5297     2818
## Intercept[2]                           5009     3112
## Intercept[3]                           4842     3325
## Intercept[4]                           5237     3193
## Intercept[5]                           5204     3135
## var_names_copied_good.ratio            6424     3457
## var_names_new_good.ratio               6238     3236
## reused_logic_validationfalse           6542     3272
## equals.existsFALSE                     5886     2923
## sonarqube_issues.s                     5454     2842
## documentationIncorrect                 5077     2676
## documentationNone                      6277     3190
## education_fieldInteractionDesign       6726     3013
## education_fieldNone                    7045     3195
## education_fieldSoftwareEngineering     6157     3265
## workplace_peer_reviewfalse             5420     2880
## moeducation_level                      4061     3120
## 
## Simplex Parameters: 
##                       Estimate Est.Error l-95% CI u-95% CI Rhat Bulk_ESS
## moeducation_level1[1]     0.22      0.13     0.03     0.50 1.00     6699
## moeducation_level1[2]     0.28      0.14     0.05     0.59 1.00     7150
## moeducation_level1[3]     0.24      0.14     0.04     0.54 1.00     6450
## moeducation_level1[4]     0.26      0.14     0.05     0.55 1.00     7011
##                       Tail_ESS
## moeducation_level1[1]     2431
## moeducation_level1[2]     2803
## moeducation_level1[3]     2737
## moeducation_level1[4]     3535
## 
## Family Specific Parameters: 
##      Estimate Est.Error l-95% CI u-95% CI Rhat Bulk_ESS Tail_ESS
## disc     1.00      0.00     1.00     1.00 1.00     4000     4000
## 
## Samples were drawn using sample(hmc). For each parameter, Bulk_ESS
## and Tail_ESS are effective sample size measures, and Rhat is the potential
## scale reduction factor on split chains (at convergence, Rhat = 1).

Random effects

ranef(own_quality3)

## $session
## , , Intercept
## 
##                             Estimate Est.Error       Q2.5     Q97.5
## 6033d69a5af2c702367b3a95  0.07491700 0.4911729 -0.9430459 1.2504775
## 6033d90a5af2c702367b3a96 -0.06422939 0.4855022 -1.2000190 0.8806904
## 6034fc165af2c702367b3a98  0.18351701 0.5363674 -0.6744247 1.6376418
## 603500725af2c702367b3a99  0.07203840 0.4972385 -0.9593952 1.2888728
## 603f97625af2c702367b3a9d  0.16653041 0.5304650 -0.7274883 1.5075223
## 603fd5d95af2c702367b3a9e  0.07430359 0.4803014 -0.8234218 1.1933480
## 60409b7b5af2c702367b3a9f -0.19549489 0.5278701 -1.5943530 0.6346203
## 604b82b5a7718fbed181b336 -0.02485149 0.4916338 -1.1153138 1.0381780
## 6050c1bf856f36729d2e5218 -0.22129918 0.5557692 -1.6922722 0.6191403
## 6050e1e7856f36729d2e5219  0.14790831 0.5310898 -0.7941579 1.5089705
## 6055fdc6856f36729d2e521b -0.09351285 0.5250341 -1.4110362 0.9013951
## 60589862856f36729d2e521f  0.03825752 0.5241489 -1.0470568 1.2377180
## 605afa3a856f36729d2e5222  0.16073087 0.5085764 -0.6865203 1.5469607
## 605c8bc6856f36729d2e5223  0.20357679 0.5468986 -0.6734016 1.6331773
## 605f3f2d856f36729d2e5224 -0.15795929 0.5378507 -1.5687513 0.7968925
## 605f46c3856f36729d2e5225 -0.20337705 0.5293703 -1.6190217 0.6239888
## 60605337856f36729d2e5226  0.02294887 0.4898817 -1.0002530 1.1326100
## 60609ae6856f36729d2e5228 -0.06817839 0.4936718 -1.2820645 0.9673343
## 6061ce91856f36729d2e522e  0.12301079 0.5143776 -0.8032908 1.4377078
## 6061f106856f36729d2e5231 -0.06028052 0.4667851 -1.1729798 0.8388732
## 6068ea9f856f36729d2e523e  0.03857722 0.5145712 -0.9961772 1.2367408
## 6075ab05856f36729d2e5247 -0.14589928 0.5416454 -1.5395385 0.8075027

Sampling plots

plot(own_quality3, ask = FALSE)

Posterior predictive check

pp_check(own_quality3, nsamples = 200, type = "bars")

Final Model

All candidate models look nice, none is significantly better than the others, we will proceed the model containing work experince as it otherwise ourd be added in the next step: own_quality0

All data points

Some participants did only complete one scenario. Those has been excluded from the initial dataset to improve sampling of the models. We do however want to use all data we can and will therefore try to fit the model with the complete dataset.

own_quality0.all <- brm(
  "quality_post_task ~ 1 + 
    var_names_copied_good.ratio + 
    var_names_new_good.ratio + 
    reused_logic_validation + 
    equals.exists + 
    sonarqube_issues.s + 
    documentation + 
    (1 | session)",
  prior = c(
    prior(normal(0, 2.5), class = "Intercept"),
    prior(normal(0, 1), class = "b"),
    prior(exponential(1), class = "sd")
  ),
  family = cumulative(),
  data = as.data.frame(d.completed),
  file = "fits/own_quality0.all",
  file_refit = "on_change",
  control = list(adapt_delta = 0.95),
  seed = 20210421
)

Summary

summary(own_quality0.all)

##  Family: cumulative 
##   Links: mu = logit; disc = identity 
## Formula: quality_post_task ~ 1 + var_names_copied_good.ratio + var_names_new_good.ratio + reused_logic_validation + equals.exists + sonarqube_issues.s + documentation + (1 | session) 
##    Data: as.data.frame(d.completed) (Number of observations: 51) 
## Samples: 4 chains, each with iter = 2000; warmup = 1000; thin = 1;
##          total post-warmup samples = 4000
## 
## Group-Level Effects: 
## ~session (Number of levels: 29) 
##               Estimate Est.Error l-95% CI u-95% CI Rhat Bulk_ESS Tail_ESS
## sd(Intercept)     0.50      0.39     0.01     1.43 1.00     1284     1779
## 
## Population-Level Effects: 
##                              Estimate Est.Error l-95% CI u-95% CI Rhat Bulk_ESS
## Intercept[1]                    -4.81      1.25    -7.39    -2.48 1.00     3799
## Intercept[2]                    -3.53      1.07    -5.71    -1.45 1.00     4382
## Intercept[3]                    -1.46      0.98    -3.43     0.45 1.00     4286
## Intercept[4]                     1.43      0.99    -0.48     3.47 1.00     4279
## Intercept[5]                     2.17      1.02     0.20     4.24 1.00     4288
## Intercept[6]                     3.12      1.11     0.99     5.41 1.00     4863
## var_names_copied_good.ratio      0.19      0.66    -1.10     1.52 1.00     4591
## var_names_new_good.ratio        -0.04      0.77    -1.53     1.46 1.00     5362
## reused_logic_validationfalse    -1.31      0.66    -2.62    -0.03 1.00     4370
## equals.existsFALSE              -0.17      0.56    -1.25     0.97 1.00     4669
## sonarqube_issues.s               0.09      0.31    -0.54     0.68 1.00     4245
## documentationIncorrect          -0.19      0.67    -1.51     1.12 1.00     4038
## documentationNone               -0.33      0.61    -1.53     0.91 1.00     4519
##                              Tail_ESS
## Intercept[1]                     2853
## Intercept[2]                     3324
## Intercept[3]                     3159
## Intercept[4]                     3267
## Intercept[5]                     3268
## Intercept[6]                     3112
## var_names_copied_good.ratio      3473
## var_names_new_good.ratio         3374
## reused_logic_validationfalse     3297
## equals.existsFALSE               2864
## sonarqube_issues.s               3069
## documentationIncorrect           3285
## documentationNone                3238
## 
## Family Specific Parameters: 
##      Estimate Est.Error l-95% CI u-95% CI Rhat Bulk_ESS Tail_ESS
## disc     1.00      0.00     1.00     1.00 1.00     4000     4000
## 
## Samples were drawn using sample(hmc). For each parameter, Bulk_ESS
## and Tail_ESS are effective sample size measures, and Rhat is the potential
## scale reduction factor on split chains (at convergence, Rhat = 1).

Random effects

ranef(own_quality0.all)

## $session
## , , Intercept
## 
##                              Estimate Est.Error       Q2.5     Q97.5
## 6033c6fc5af2c702367b3a93 -0.349289242 0.7795327 -2.5428940 0.6959831
## 6033d69a5af2c702367b3a95  0.026530756 0.5260593 -1.0998937 1.2147825
## 6033d90a5af2c702367b3a96  0.029703767 0.5164378 -1.0819540 1.2027353
## 6034fc165af2c702367b3a98  0.238125196 0.6090597 -0.7396881 1.8299838
## 603500725af2c702367b3a99  0.231422198 0.5973404 -0.7560744 1.7214345
## 603f84f15af2c702367b3a9b  0.005333960 0.5695396 -1.2383455 1.2915118
## 603f97625af2c702367b3a9d  0.210284903 0.5875872 -0.7253746 1.7977908
## 603fd5d95af2c702367b3a9e  0.223026533 0.5820774 -0.7166333 1.7001420
## 60409b7b5af2c702367b3a9f -0.193239496 0.5464399 -1.5337175 0.7521659
## 604b82b5a7718fbed181b336 -0.067691678 0.5589897 -1.3786600 1.0439938
## 604f1239a7718fbed181b33f  0.015989792 0.5593005 -1.2059358 1.2437528
## 6050c1bf856f36729d2e5218 -0.273141168 0.5758688 -1.7497000 0.6201582
## 6050e1e7856f36729d2e5219  0.278264995 0.6353341 -0.6588195 1.9282408
## 6055fdc6856f36729d2e521b -0.053030916 0.5364673 -1.3167242 1.0154745
## 60579f2a856f36729d2e521e -0.218149016 0.6219092 -1.8630270 0.7897970
## 60589862856f36729d2e521f  0.039798725 0.5767338 -1.1643550 1.3791355
## 605a30a7856f36729d2e5221  0.001669608 0.5450172 -1.1957662 1.1962413
## 605afa3a856f36729d2e5222  0.122957724 0.5181212 -0.8428584 1.3465340
## 605c8bc6856f36729d2e5223  0.130266187 0.5412859 -0.9127264 1.4678885
## 605f3f2d856f36729d2e5224 -0.200733573 0.5628212 -1.6064560 0.7337097
## 605f46c3856f36729d2e5225 -0.343036209 0.6548436 -2.1283313 0.5793587
## 60605337856f36729d2e5226 -0.004899536 0.5175498 -1.1205360 1.1506197
## 60609ae6856f36729d2e5228 -0.070159489 0.5107317 -1.2259107 1.0033435
## 6061ce91856f36729d2e522e  0.242088562 0.5935935 -0.6770190 1.8501147
## 6061f106856f36729d2e5231  0.071695997 0.5207193 -0.9846461 1.2769810
## 60672faa856f36729d2e523c  0.037295031 0.5385528 -1.0737692 1.2501955
## 6068ea9f856f36729d2e523e  0.215376027 0.5700790 -0.6903914 1.6900623
## 606db69d856f36729d2e5243  0.005385041 0.5639127 -1.2658303 1.2829358
## 6075ab05856f36729d2e5247 -0.365414846 0.6426471 -2.0586880 0.4860709

Sampling plots

plot(own_quality0.all, ask = FALSE)

Posterior predictive check

pp_check(own_quality0.all, nsamples = 200, type = "bars")

Final model

• Fitting the model to all data point did not significantly damage the model and will be used as is a more fair representation of reality.

This means that our final model, with all data points and experience predictors, is own_quality0.all

Interpreting the model

To begin interpreting the model we look at how it’s parameters were estimated. As our research is focused on how the outcome of the model is effected we will mainly analyze the \(\beta\) parameters.

\(\beta\) parameters

mcmc_areas(own_quality0.all, 
           pars = c(
             "b_var_names_copied_good.ratio", 
             "b_var_names_new_good.ratio", 
             "b_reused_logic_validationfalse",
             "b_equals.existsFALSE",
             "b_sonarqube_issues.s",
             "b_documentationIncorrect",
             "b_documentationNone"
                    ),
           prob = 0.95) + scale_y_discrete() +
  scale_y_discrete(labels=c(
    "Ratio of good copied var names",
    "Ratio of good new var names",
    "Duplicated validation logic",
    "Missing equals implementation",
    "Amount of sonarqube issues",
    "Incorrect documentation",
    "No documentation"
    )) +
  ggtitle("Beta parameters densities in self assesed quality model", subtitle = "Shaded region marks 95% of the density. Line marks the median")

As we have a low of effects playing small roles we will simulate two scenarios, one where the developer, according to us does well and one where the developers doesnt do so well and se if the participant rating approves with us.

scale_programming_experience <- function(x) {
  (x - mean(d.completed$work_experience_programming))/ sd(d.completed$work_experience_programming)
}
unscale_programming_experience <- function(x) {
  x * sd(d.completed$work_experience_programming) + mean(d.completed$work_experience_programming)
}

post_settings <- data_frame(
  var_names_copied_good.ratio = c(0.9, 0.5),
  var_names_new_good.ratio = c(0.9, 0.5),
  reused_logic_validation = c("true", "false"),
  equals.exists = c("TRUE", "FALSE"),
  sonarqube_issues.s = c(-1, 1),
  documentation = c("Correct", "Incorrect"),
  session = NA
)

post <- posterior_predict(own_quality0.all, newdata = post_settings) %>%
  melt(value.name = "estimate", varnames = c("sample_number", "settings_id")) %>%
  left_join(
    rowid_to_column(post_settings, var= "settings_id"),
    by = "settings_id"
  ) %>%
  mutate(submission = revalue(reused_logic_validation, c("true" = "Good", "false" = "Bad"))) %>%
  select(
    estimate,
    submission
  )

post.nice <- post %>%  mutate_at("estimate", function(x) revalue(as.ordered(x), c(
      "1"="Very Bad",
      "2"="Bad",
      "3"="Somewhat Bad",
      "4"="Neutral",
      "5"="Somewhat Good",
      "6"="Good",
      "7"="Very Good"
    )))

data.frame(
  Bad_submission = post.nice %>%
    filter(submission == "Bad") %>%
    pull(estimate),
  Good_submission = post.nice %>%
    filter(submission == "Good") %>%
    pull(estimate)
) %>%
  likert() %>%
  plot(
    type="density",
    facet = TRUE,
  )